/**
 * Alipay.com Inc.
 * Copyright (c) 2004-2022 All Rights Reserved.
 */
package com.example.spider.proxy.provider.xsdaili;

import cn.hutool.core.util.ReUtil;
import com.example.spider.proxy.provider.AbstractProxyResultHandler;
import com.example.spider.proxy.provider.IpProxySpiderTask;
import com.example.spider.starter.SpiderStarter;
import com.example.spider.task.AbstractSpiderTask;
import com.example.spider.task.SpiderTaskQueue;
import com.example.spider.util.FileBasedCollection;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;

import java.time.Duration;
import java.time.LocalDate;
import java.util.List;
import java.util.concurrent.ConcurrentSkipListSet;

/**
 * 小舒代理 List
 * 这个代理网站代理需要去重
 *
 * @author lym
 */
//@Component
public class XsDailiListHandler extends AbstractProxyResultHandler implements SpiderStarter {

    @Autowired
    @Qualifier("proxy_xsdaili_distinct_set")
    private FileBasedCollection<ConcurrentSkipListSet<String>, String> distinctSet;

    public XsDailiListHandler() {
        super("xsDaili-list");
    }

    protected Duration defaultLimitRateDuration() {
        return Duration.ofMillis(1000);
    }

    @Override
    public void startSpiders() {
        LocalDate now = LocalDate.now();
        int year = now.getYear();
        int month = now.getMonthValue();
        int day = now.getDayOfMonth();

        Object[] param = new Object[1];
        param[0] = 1;
        // 去年一整年
//        for (int monthIndex = 1; monthIndex < 12; monthIndex++) {
//            AbstractSpiderTask<String> task = createTask("https://www.xsdaili.cn/dayProxy/" + (year - 1) + "/" + monthIndex + "/1.html")
//                    .setParams(param);
//            SpiderTaskQueue.putTask(task);
//        }
        for (int monthIndex = 1; monthIndex < month; monthIndex++) {
            AbstractSpiderTask<String> task = createTask("https://www.xsdaili.cn/dayProxy/" + year + "/" + monthIndex + "/1.html")
                    .setParams(param)
                    .setIdempotentFunction(t -> {
                        if (distinctSet.contains(t.getUrl().split("www.xsdaili.cn")[1])) {
                            return "REPEAT";
                        }
                        return null;
                    });
            SpiderTaskQueue.putTask(task);
        }
    }

    /**
     * 每天 2. 触发一次
     */
    @Override
    public String executeCron() {
        return "0 0 2 * * ?";
    }


    /**
     * 处理响应
     */
    @Override
    public void doHandle(IpProxySpiderTask task, String html) {

        String regex = "<a href=\"(/dayProxy/ip/\\d+.html)\">";

        List<String> urlList = ReUtil.findAll(regex, html, 1);

        for (int i = 0; i < urlList.size(); i++) {
            // 爬详情
            IpProxySpiderTask t = createTask("https://www.xsdaili.cn" + urlList.get(i))
                    .setIpProxyTaskId("xsDaili")
                    .setIdempotentFunction(x -> {
                        if (distinctSet.contains(x.getUrl().split("www.xsdaili.cn")[1])) {
                            return "REPEAT";
                        }
                        return null;
                    });
            SpiderTaskQueue.putTask(t);
        }
        distinctSet.add(task.getUrl().split("www.xsdaili.cn")[1]);
        if (html.contains("下一页")) {
            int page = (int) task.getParams()[0];
            Object[] param = new Object[1];
            param[0] = page + 1;
            AbstractSpiderTask<String> t = createTask(task.getUrl().replace(page + ".html", param[0] + ".html"))
                    .setParams(param)
                    .setIdempotentFunction(x -> {
                        if (distinctSet.contains(x.getUrl().split("www.xsdaili.cn")[1])) {
                            return "REPEAT";
                        }
                        return null;
                    });
            SpiderTaskQueue.putTask(t);
        }
    }

}